{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cefda1e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "\n",
    "SOURCE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__name__))))\n",
    "SOURCE_DIR\n",
    "sys.path.insert(0, SOURCE_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.08 s, sys: 3.17 s, total: 6.24 s\n",
      "Wall time: 2.46 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<module 'malaya' from '/home/husein/dev/malaya/malaya/__init__.py'>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "import malaya\n",
    "malaya"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalizer = malaya.normalize.normalizer(date = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "segmenter = malaya.segmentation.huggingface()\n",
    "segmenter_func = lambda x: segmenter.generate([x], max_length = 256)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You're using a T5TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 677 ms, sys: 4.98 ms, total: 682 ms\n",
      "Wall time: 64.2 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'tak kisah'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "segmenter_func('takkisah')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
     ]
    }
   ],
   "source": [
    "fasttext = malaya.language_detection.fasttext()\n",
    "lang_word_model = malaya.language_detection.substring_rules(model = fasttext)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "\n",
    "twitters = glob('/home/husein/malaya/2022-06-08/*.json')\n",
    "twitters.extend(glob('/home/husein/malaya/compiled/*.json'))\n",
    "twitters = sorted(twitters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/home/husein/malaya/2022-06-08/2022-06-08-bahasa-0.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-1.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-10.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-11.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-12.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-13.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-14.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-15.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-16.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-17.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-18.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-19.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-2.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-20.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-21.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-22.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-23.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-24.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-25.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-3.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-4.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-5.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-6.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-7.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-8.json',\n",
       " '/home/husein/malaya/2022-06-08/2022-06-08-bahasa-9.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-0.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-1.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-10.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-11.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-12.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-13.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-14.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-15.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-16.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-17.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-18.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-19.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-2.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-20.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-21.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-22.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-23.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-24.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-25.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-26.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-27.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-28.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-29.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-3.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-30.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-31.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-32.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-33.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-34.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-35.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-36.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-37.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-38.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-39.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-4.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-40.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-41.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-5.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-6.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-7.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-8.json',\n",
       " '/home/husein/malaya/compiled/2021-dump-bahasa-twitter-9.json']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "rejected_words = ['sik', 'aktivitas', 'lanat', 'keren', 'loh', 'akh', 'pake', 'goblok', 'ivo',\n",
    "                 'listrik', 'ludes', 'pemudik', 'stasiun', 'ketidaknyamananya', 'gawe', 'jempol',\n",
    "                 'kesahatan', 'sivitas', 'ngasih', 'pasokan', 'esdm', 'miliar', 'domisili',\n",
    "                 'nyengine', 'indonesia', 'ndak', 'mbak', 'gimana', 'verif']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "def query(text, from_lang = 'en', to_lang = 'ms'):\n",
    "    return requests.get(f'http://100.105.246.81:8999/api?from={from_lang}&to={to_lang}&text={text}').json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_scorer = lambda x: lm.score(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def fix_spacing_punct(string):\n",
    "    string = string.replace(' , ', ', ').replace(' !', '!').replace(' . ', '. ').replace('( ', '(') \\\n",
    "    .replace(' )', ')')\n",
    "    return re.sub(r'[ ]+', ' ', string).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-0.json 2022-06-08-bahasa-0.json.v2.normalized 2022-06-08-bahasa-0.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:38<00:00, 2625.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-1.json 2022-06-08-bahasa-1.json.v2.normalized 2022-06-08-bahasa-1.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:42<00:00, 2351.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-10.json 2022-06-08-bahasa-10.json.v2.normalized 2022-06-08-bahasa-10.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:45<00:00, 2192.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-11.json 2022-06-08-bahasa-11.json.v2.normalized 2022-06-08-bahasa-11.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:42<00:00, 2350.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-12.json 2022-06-08-bahasa-12.json.v2.normalized 2022-06-08-bahasa-12.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:41<00:00, 2407.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-13.json 2022-06-08-bahasa-13.json.v2.normalized 2022-06-08-bahasa-13.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:42<00:00, 2371.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-14.json 2022-06-08-bahasa-14.json.v2.normalized 2022-06-08-bahasa-14.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:42<00:00, 2332.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-15.json 2022-06-08-bahasa-15.json.v2.normalized 2022-06-08-bahasa-15.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:42<00:00, 2363.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-16.json 2022-06-08-bahasa-16.json.v2.normalized 2022-06-08-bahasa-16.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:41<00:00, 2393.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-17.json 2022-06-08-bahasa-17.json.v2.normalized 2022-06-08-bahasa-17.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:43<00:00, 2288.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-18.json 2022-06-08-bahasa-18.json.v2.normalized 2022-06-08-bahasa-18.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:40<00:00, 2493.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-19.json 2022-06-08-bahasa-19.json.v2.normalized 2022-06-08-bahasa-19.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:42<00:00, 2350.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-2.json 2022-06-08-bahasa-2.json.v2.normalized 2022-06-08-bahasa-2.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:42<00:00, 2358.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-20.json 2022-06-08-bahasa-20.json.v2.normalized 2022-06-08-bahasa-20.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:45<00:00, 2193.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-21.json 2022-06-08-bahasa-21.json.v2.normalized 2022-06-08-bahasa-21.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:41<00:00, 2415.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-22.json 2022-06-08-bahasa-22.json.v2.normalized 2022-06-08-bahasa-22.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:43<00:00, 2300.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-23.json 2022-06-08-bahasa-23.json.v2.normalized 2022-06-08-bahasa-23.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:38<00:00, 2590.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-24.json 2022-06-08-bahasa-24.json.v2.normalized 2022-06-08-bahasa-24.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:40<00:00, 2452.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-25.json 2022-06-08-bahasa-25.json.v2.normalized 2022-06-08-bahasa-25.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 79802/79802 [00:31<00:00, 2508.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-3.json 2022-06-08-bahasa-3.json.v2.normalized 2022-06-08-bahasa-3.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:37<00:00, 2633.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-4.json 2022-06-08-bahasa-4.json.v2.normalized 2022-06-08-bahasa-4.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:42<00:00, 2338.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-5.json 2022-06-08-bahasa-5.json.v2.normalized 2022-06-08-bahasa-5.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:43<00:00, 2284.79it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-6.json 2022-06-08-bahasa-6.json.v2.normalized 2022-06-08-bahasa-6.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:42<00:00, 2378.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-7.json 2022-06-08-bahasa-7.json.v2.normalized 2022-06-08-bahasa-7.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:40<00:00, 2457.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-8.json 2022-06-08-bahasa-8.json.v2.normalized 2022-06-08-bahasa-8.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:40<00:00, 2484.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-06-08-bahasa-9.json 2022-06-08-bahasa-9.json.v2.normalized 2022-06-08-bahasa-9.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:42<00:00, 2355.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-0.json 2021-dump-bahasa-twitter-0.json.v2.normalized 2021-dump-bahasa-twitter-0.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:35<00:00, 2788.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-1.json 2021-dump-bahasa-twitter-1.json.v2.normalized 2021-dump-bahasa-twitter-1.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:37<00:00, 2674.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-10.json 2021-dump-bahasa-twitter-10.json.v2.normalized 2021-dump-bahasa-twitter-10.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:35<00:00, 2780.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-11.json 2021-dump-bahasa-twitter-11.json.v2.normalized 2021-dump-bahasa-twitter-11.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:41<00:00, 2416.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-12.json 2021-dump-bahasa-twitter-12.json.v2.normalized 2021-dump-bahasa-twitter-12.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:41<00:00, 2399.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-13.json 2021-dump-bahasa-twitter-13.json.v2.normalized 2021-dump-bahasa-twitter-13.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:40<00:00, 2486.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-14.json 2021-dump-bahasa-twitter-14.json.v2.normalized 2021-dump-bahasa-twitter-14.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:35<00:00, 2837.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-15.json 2021-dump-bahasa-twitter-15.json.v2.normalized 2021-dump-bahasa-twitter-15.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:31<00:00, 3159.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-16.json 2021-dump-bahasa-twitter-16.json.v2.normalized 2021-dump-bahasa-twitter-16.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:34<00:00, 2859.63it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-17.json 2021-dump-bahasa-twitter-17.json.v2.normalized 2021-dump-bahasa-twitter-17.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:36<00:00, 2747.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-18.json 2021-dump-bahasa-twitter-18.json.v2.normalized 2021-dump-bahasa-twitter-18.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:35<00:00, 2785.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-19.json 2021-dump-bahasa-twitter-19.json.v2.normalized 2021-dump-bahasa-twitter-19.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:36<00:00, 2734.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-2.json 2021-dump-bahasa-twitter-2.json.v2.normalized 2021-dump-bahasa-twitter-2.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:38<00:00, 2599.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-20.json 2021-dump-bahasa-twitter-20.json.v2.normalized 2021-dump-bahasa-twitter-20.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:36<00:00, 2728.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-21.json 2021-dump-bahasa-twitter-21.json.v2.normalized 2021-dump-bahasa-twitter-21.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:38<00:00, 2577.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-22.json 2021-dump-bahasa-twitter-22.json.v2.normalized 2021-dump-bahasa-twitter-22.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:36<00:00, 2731.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-23.json 2021-dump-bahasa-twitter-23.json.v2.normalized 2021-dump-bahasa-twitter-23.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:37<00:00, 2686.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-24.json 2021-dump-bahasa-twitter-24.json.v2.normalized 2021-dump-bahasa-twitter-24.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:33<00:00, 3011.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-25.json 2021-dump-bahasa-twitter-25.json.v2.normalized 2021-dump-bahasa-twitter-25.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:36<00:00, 2768.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-26.json 2021-dump-bahasa-twitter-26.json.v2.normalized 2021-dump-bahasa-twitter-26.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:37<00:00, 2647.77it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-27.json 2021-dump-bahasa-twitter-27.json.v2.normalized 2021-dump-bahasa-twitter-27.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:35<00:00, 2851.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-28.json 2021-dump-bahasa-twitter-28.json.v2.normalized 2021-dump-bahasa-twitter-28.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:37<00:00, 2677.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-29.json 2021-dump-bahasa-twitter-29.json.v2.normalized 2021-dump-bahasa-twitter-29.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:36<00:00, 2754.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-3.json 2021-dump-bahasa-twitter-3.json.v2.normalized 2021-dump-bahasa-twitter-3.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:37<00:00, 2701.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-30.json 2021-dump-bahasa-twitter-30.json.v2.normalized 2021-dump-bahasa-twitter-30.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:39<00:00, 2505.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-31.json 2021-dump-bahasa-twitter-31.json.v2.normalized 2021-dump-bahasa-twitter-31.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:31<00:00, 3169.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-32.json 2021-dump-bahasa-twitter-32.json.v2.normalized 2021-dump-bahasa-twitter-32.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:33<00:00, 2970.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-33.json 2021-dump-bahasa-twitter-33.json.v2.normalized 2021-dump-bahasa-twitter-33.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:35<00:00, 2788.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-34.json 2021-dump-bahasa-twitter-34.json.v2.normalized 2021-dump-bahasa-twitter-34.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:39<00:00, 2533.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-35.json 2021-dump-bahasa-twitter-35.json.v2.normalized 2021-dump-bahasa-twitter-35.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:40<00:00, 2446.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-36.json 2021-dump-bahasa-twitter-36.json.v2.normalized 2021-dump-bahasa-twitter-36.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:34<00:00, 2863.45it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-37.json 2021-dump-bahasa-twitter-37.json.v2.normalized 2021-dump-bahasa-twitter-37.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:33<00:00, 3011.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-38.json 2021-dump-bahasa-twitter-38.json.v2.normalized 2021-dump-bahasa-twitter-38.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:32<00:00, 3109.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-39.json 2021-dump-bahasa-twitter-39.json.v2.normalized 2021-dump-bahasa-twitter-39.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:39<00:00, 2513.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-4.json 2021-dump-bahasa-twitter-4.json.v2.normalized 2021-dump-bahasa-twitter-4.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:35<00:00, 2802.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-40.json 2021-dump-bahasa-twitter-40.json.v2.normalized 2021-dump-bahasa-twitter-40.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:43<00:00, 2303.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-41.json 2021-dump-bahasa-twitter-41.json.v2.normalized 2021-dump-bahasa-twitter-41.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:38<00:00, 2589.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-5.json 2021-dump-bahasa-twitter-5.json.v2.normalized 2021-dump-bahasa-twitter-5.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:36<00:00, 2725.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-6.json 2021-dump-bahasa-twitter-6.json.v2.normalized 2021-dump-bahasa-twitter-6.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:34<00:00, 2897.15it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-7.json 2021-dump-bahasa-twitter-7.json.v2.normalized 2021-dump-bahasa-twitter-7.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:34<00:00, 2861.06it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-8.json 2021-dump-bahasa-twitter-8.json.v2.normalized 2021-dump-bahasa-twitter-8.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:38<00:00, 2572.11it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2021-dump-bahasa-twitter-9.json 2021-dump-bahasa-twitter-9.json.v2.normalized 2021-dump-bahasa-twitter-9.v2.normalized.jsonl.requested\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████| 100000/100000 [00:37<00:00, 2671.38it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import traceback\n",
    "import json\n",
    "\n",
    "for f in twitters:\n",
    "    filename = os.path.split(f)[1]\n",
    "    output = f'{filename}.v2.normalized'\n",
    "    output_requested = f\"{filename.replace('.json', '')}.v2.normalized.jsonl.requested\"\n",
    "    print(filename, output, output_requested)\n",
    "    \n",
    "    if os.path.exists(output_requested):\n",
    "        print(f'skip {filename}')\n",
    "        continue\n",
    "    \n",
    "    with open(f) as fopen:\n",
    "        data = json.load(fopen)\n",
    "\n",
    "    results = []\n",
    "    already_in = set()\n",
    "    for row in tqdm(data):\n",
    "        if row['retweet_text_full'] == 'NULL':\n",
    "            t = row['data_text']\n",
    "        else:\n",
    "            t = row['retweet_text_full']\n",
    "\n",
    "        if fasttext.predict([t])[0] not in ['malay', 'rojak']:\n",
    "            continue\n",
    "\n",
    "        if t.lower() in already_in:\n",
    "            continue\n",
    "\n",
    "        tokenized = normalizer._tokenizer(t.lower())\n",
    "        if any([w in tokenized for w in rejected_words]):\n",
    "            continue\n",
    "\n",
    "        if not len(t):\n",
    "            continue\n",
    "\n",
    "        try:\n",
    "            normalized = normalizer.normalize(\n",
    "                t, normalize_hingga = False, normalize_cardinal = False,\n",
    "                normalize_ordinal = False, normalize_pada_hari_bulan = False,\n",
    "                normalize_fraction = False, normalize_money = False, normalize_date = False,\n",
    "                normalize_time = False, normalize_ic = False,\n",
    "                normalize_units = False,\n",
    "                normalize_url = False, normalize_percent=False, normalize_telephone = False,\n",
    "                not_a_word_threshold = 1e-9,\n",
    "            )['normalize']\n",
    "            \n",
    "            normalized = fix_spacing_punct(normalized)\n",
    "            \n",
    "        except Exception as e:\n",
    "            print(traceback.format_exc())\n",
    "            print(e)\n",
    "            normalized = t\n",
    "        \n",
    "        results.append({'original': t, 'normalized': normalized})\n",
    "        already_in.add(t.lower())\n",
    "    \n",
    "    with open(output, 'w') as fopen:\n",
    "        json.dump(results, fopen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fix_spacing_punct(results[5]['normalized'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalizer._tokenizer('RM35.92 . Get')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = malaya.tokenizer.Tokenizer(parliament = True).tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer('RM35.92 . Get')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer('RM35.92. Get')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
